const fs = require('fs-extra');
const axios = require('axios');
const cheerio = require('cheerio');
const moment = require('moment');

/**
 * 增强版只看楼主爬取器
 * @description 专门解决楼主识别和数据完整性问题
 */
class EnhancedOPCrawler {
  /**
   * 构造函数
   * @param {Object} options - 配置选项
   */
  constructor(options = {}) {
    this.cookies = options.cookies || '';
    this.maxPages = options.maxPages || 200; // 大幅增加页数
    this.baseDelay = options.delay || 4000; // 增加基础延迟
    this.maxRetries = 3;
    this.userAgents = [
      'Mozilla/5.0 (iPhone; CPU iPhone OS 15_0 like Mac OS X) AppleWebKit/605.1.15',
      'Mozilla/5.0 (Android 11; Mobile; rv:68.0) Gecko/68.0 Firefox/88.0',
      'Mozilla/5.0 (Linux; Android 10; SM-G973F) AppleWebKit/537.36'
    ];
    
    this.initHttpClient();
  }

  /**
   * 初始化HTTP客户端
   */
  initHttpClient() {
    this.httpClient = axios.create({
      timeout: 25000,
      headers: {
        'User-Agent': this.getRandomUserAgent(),
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
        'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
        'Referer': 'https://waptieba.baidu.com/',
        'Cookie': this.cookies
      }
    });
  }

  /**
   * 获取随机User-Agent
   */
  getRandomUserAgent() {
    return this.userAgents[Math.floor(Math.random() * this.userAgents.length)];
  }

  /**
   * 延迟方法
   */
  async sleep(ms) {
    return new Promise(resolve => setTimeout(resolve, ms));
  }

  /**
   * 主要爬取方法
   */
  async crawlCompleteOP(postUrl, outputName = 'enhanced_op_complete') {
    try {
      console.log('🚀 启动增强版只看楼主完整爬取器...');
      
      const wapUrl = this.convertToWapUrl(postUrl);
      console.log(`🔗 目标URL: ${wapUrl}`);
      
      // 第一步：获取帖子基本信息
      const postInfo = await this.analyzePostInfo(wapUrl);
      console.log(`📋 帖子: ${postInfo.title}`);
      console.log(`👑 楼主: ${postInfo.opName}`);
      
      // 第二步：使用多策略爬取完整楼主发言
      const allOPPosts = await this.crawlWithMultiStrategy(wapUrl, postInfo);
      
      // 第三步：数据清理和排序
      const cleanedPosts = this.cleanAndSortPosts(allOPPosts);
      
      // 第四步：生成完整报告
      const finalReport = this.generateCompleteReport(postInfo, cleanedPosts);
      
      // 第五步：保存结果
      const savedFiles = await this.saveResults(finalReport, outputName);
      
      return {
        success: true,
        postInfo: postInfo,
        totalPosts: cleanedPosts.length,
        totalWords: cleanedPosts.reduce((sum, p) => sum + p.字数, 0),
        savedFiles: savedFiles
      };
      
    } catch (error) {
      console.error('❌ 增强爬取失败:', error.message);
      throw error;
    }
  }

  /**
   * 分析帖子信息
   */
  async analyzePostInfo(wapUrl) {
    console.log('📊 分析帖子基本信息...');
    
    const response = await this.httpClient.get(wapUrl);
    const $ = cheerio.load(response.data);
    
    // 提取标题
    const title = this.extractTitle($);
    
    // 提取楼主名称 - 多种方法
    const opName = this.extractOPName($);
    
    // 提取帖子ID
    const postId = wapUrl.match(/\/p\/(\d+)/)?.[1] || Date.now().toString();
    
    return {
      title: this.cleanText(title),
      opName: opName,
      postId: postId,
      originalUrl: wapUrl.replace('waptieba.baidu.com', 'tieba.baidu.com'),
      wapUrl: wapUrl
    };
  }

  /**
   * 提取标题
   */
  extractTitle($) {
    const selectors = ['title', '.post_title', 'h1', '.pb_title'];
    for (const selector of selectors) {
      const title = $(selector).text().trim();
      if (title) return title;
    }
    return '未知标题';
  }

  /**
   * 提取楼主名称
   */
  extractOPName($) {
    // 方法1: 从楼主标识提取
    const opFlag = $('.j_floor_lz.floor_lz_icon');
    if (opFlag.length > 0) {
      // 找到楼主标识的父级容器，然后找用户名
      const $container = opFlag.closest('.list_item, .list_item_wrapper');
      if ($container.length > 0) {
        const author = $container.find('.user_name').text().trim();
        if (author) {
          console.log(`  ✅ 从楼主标识识别: ${author}`);
          return author;
        }
      }
    }
    
    // 方法2: 从具有lz="1"属性的内容区域向上找用户名
    const $lzContent = $('[lz="1"]').first();
    if ($lzContent.length > 0) {
      const $container = $lzContent.closest('.list_item, .list_item_wrapper');
      if ($container.length > 0) {
        const author = $container.find('.user_name').text().trim();
        if (author) {
          console.log(`  ✅ 从lz属性识别: ${author}`);
          return author;
        }
      }
    }
    
    // 方法3: 从第一个帖子提取
    const firstPost = $('.list_item').first();
    if (firstPost.length > 0) {
      const author = firstPost.find('.user_name').text().trim();
      if (author) {
        console.log(`  ✅ 从第一帖识别: ${author}`);
        return author;
      }
    }
    
    return '楼主';
  }

  /**
   * 提取作者
   */
  extractAuthor($post) {
    const selectors = ['.user_name', '.d_name', '.username', '.author'];
    for (const selector of selectors) {
      const author = $post.find(selector).text().trim();
      if (author) return this.cleanText(author);
    }
    return null;
  }

  /**
   * 多策略爬取
   */
  async crawlWithMultiStrategy(wapUrl, postInfo) {
    console.log('📖 开始多策略完整爬取...');
    
    let allPosts = [];
    let currentPage = 1;
    let emptyCount = 0;
    let totalScanned = 0;
    let maxPageDetected = null; // 检测到的最大页数
    
    while (currentPage <= this.maxPages && emptyCount < 3) { // 减少空页容忍度
      try {
        console.log(`  📄 扫描第${currentPage}页...`);
        
        // 构建只看楼主URL
        const opUrl = this.buildOPUrl(wapUrl, currentPage);
        
        // 动态延迟
        const delay = this.calculateDelay(currentPage);
        await this.sleep(delay);
        
        // 更新User-Agent
        this.httpClient.defaults.headers['User-Agent'] = this.getRandomUserAgent();
        
        const response = await this.httpClient.get(opUrl);
        const $ = cheerio.load(response.data);
        
        // 检测总页数（只在第一页检测）
        if (currentPage === 1 && !maxPageDetected) {
          maxPageDetected = this.detectTotalPages($, response.data);
          if (maxPageDetected) {
            console.log(`  📊 检测到总页数: ${maxPageDetected}页`);
            // 更新最大页数限制
            this.maxPages = Math.min(this.maxPages, maxPageDetected);
          }
        }
        
        // 提取该页楼主发言
        const pagePosts = this.extractPagePosts($, currentPage);
        
        totalScanned++;
        
        if (pagePosts.length === 0) {
          emptyCount++;
          console.log(`    ⚠️  第${currentPage}页无发言 (空页计数: ${emptyCount})`);
          
          // 如果已经超过检测到的总页数，直接结束
          if (maxPageDetected && currentPage >= maxPageDetected) {
            console.log(`    🏁 已超过总页数(${maxPageDetected})，停止爬取`);
            break;
          }
        } else {
          emptyCount = 0;
          allPosts.push(...pagePosts);
          console.log(`    ✅ 第${currentPage}页获取${pagePosts.length}条发言`);
        }
        
        // 检查是否有下一页
        if (!this.hasNextPage($)) {
          console.log('    🏁 检测到分页结束标志');
          break;
        }
        
        // 如果已经到达检测到的最大页数，停止
        if (maxPageDetected && currentPage >= maxPageDetected) {
          console.log(`    🏁 已到达最大页数(${maxPageDetected})，停止爬取`);
          break;
        }
        
        currentPage++;
        
      } catch (error) {
        console.error(`  ❌ 第${currentPage}页失败: ${error.message}`);
        emptyCount++;
        
        // 如果连续失败且已超过预期页数，直接停止
        if (emptyCount >= 3 || (maxPageDetected && currentPage > maxPageDetected)) {
          console.log('连续失败过多或超过预期页数，停止爬取');
          break;
        }
        
        await this.sleep(this.baseDelay * 2);
        currentPage++;
      }
    }
    
    console.log(`📊 爬取统计: 共扫描${totalScanned}页，获取${allPosts.length}条楼主发言`);
    if (maxPageDetected) {
      console.log(`📋 页数信息: 检测到${maxPageDetected}页，实际扫描${totalScanned}页`);
    }
    return allPosts;
  }

  /**
   * 检测总页数 - 修复版
   * @param {Object} $ - cheerio对象
   * @param {string} html - 完整HTML内容
   * @returns {number|null} - 总页数
   */
  detectTotalPages($, html = '') {
    console.log('🔍 检测总页数...');
    
    // 方法1: 从JavaScript配置对象中提取 (最准确)
    try {
      const jsMatches = [
        /"page":\s*\{[^}]*"total_page":\s*(\d+)[^}]*\}/,
        /"total_page":\s*(\d+)/,
        /total_page["']:\s*(\d+)/,
        /totalPage["']?\s*:\s*(\d+)/
      ];
      
      for (const pattern of jsMatches) {
        const match = html.match(pattern);
        if (match) {
          const totalPage = parseInt(match[1]);
          console.log(`  ✅ 从JS配置提取总页数: ${totalPage}`);
          return totalPage;
        }
      }
    } catch (e) {
      console.log('  ⚠️  JS配置解析失败:', e.message);
    }

    // 方法2: 查找"共X页"文本
    const pageTextMatches = [
      /共(\d+)页/,
      /总共(\d+)页/,
      /Total:\s*(\d+)\s*pages?/i
    ];
    
    for (const pattern of pageTextMatches) {
      const match = html.match(pattern);
      if (match) {
        const totalPage = parseInt(match[1]);
        console.log(`  ✅ 从文本提取总页数: ${totalPage}`);
        return totalPage;
      }
    }
    
    // 方法3: 查找分页链接中的最大页码
    let maxPage = 0;
    $('a[href*="pn="]').each((i, el) => {
      const href = $(el).attr('href');
      const pnMatch = href.match(/pn=(\d+)/);
      if (pnMatch) {
        const pageNum = parseInt(pnMatch[1]);
        if (pageNum > maxPage) {
          maxPage = pageNum;
        }
      }
    });
    
    if (maxPage > 0) {
      console.log(`  ✅ 从分页链接提取最大页数: ${maxPage}`);
      return maxPage;
    }
    
    // 方法4: 查找"尾页"链接
    const lastPageLink = $('a:contains("尾页"), a:contains("末页")').first();
    if (lastPageLink.length > 0) {
      const href = lastPageLink.attr('href');
      const pnMatch = href.match(/pn=(\d+)/);
      if (pnMatch) {
        const totalPage = parseInt(pnMatch[1]);
        console.log(`  ✅ 从尾页链接提取总页数: ${totalPage}`);
        return totalPage;
      }
    }
    
    console.log('  ❌ 无法检测总页数');
    return null;
  }

  /**
   * 检查下一页 - 优化版
   */
  hasNextPage($) {
    // 方法1: 检查"下一页"链接
    const nextSelectors = [
      'a[href*="pn="]:contains("下一页")',
      'a[href*="pn="]:contains("下页")', 
      'a[href*="pn="]:contains(">")',
      'a[href*="pn="]:contains("Next")'
    ];
    
    for (const selector of nextSelectors) {
      const $nextLink = $(selector);
      if ($nextLink.length > 0) {
        // 确保链接不是disabled状态
        if (!$nextLink.hasClass('disabled') && !$nextLink.closest('li').hasClass('disabled')) {
          return true;
        }
      }
    }
    
    // 方法2: 检查当前页码是否小于总页数
    const currentPageInfo = this.getCurrentPageInfo($);
    if (currentPageInfo.current && currentPageInfo.total) {
      return currentPageInfo.current < currentPageInfo.total;
    }
    
    // 方法3: 检查是否有更多内容的指示器
    const moreIndicators = [
      ':contains("更多楼主发言")',
      ':contains("加载更多")'
    ];
    
    for (const indicator of moreIndicators) {
      if ($(indicator).length > 0) return true;
    }
    
    return false;
  }

  /**
   * 获取当前页面信息
   * @param {Object} $ - cheerio对象
   * @returns {Object} - 页面信息 {current, total}
   */
  getCurrentPageInfo($) {
    const pageText = $.html();
    
    // 查找"第X页/共Y页"格式
    const pageMatch = pageText.match(/第(\d+)页.*?共(\d+)页/);
    if (pageMatch) {
      return {
        current: parseInt(pageMatch[1]),
        total: parseInt(pageMatch[2])
      };
    }
    
    // 查找"X/Y"格式
    const slashMatch = pageText.match(/(\d+)\/(\d+)/);
    if (slashMatch) {
      return {
        current: parseInt(slashMatch[1]),
        total: parseInt(slashMatch[2])
      };
    }
    
    return { current: null, total: null };
  }

  /**
   * 构建只看楼主URL - 修复版
   * @param {string} wapUrl - WAP版本URL 
   * @param {number} page - 页码
   * @returns {string} - 只看楼主URL
   */
  buildOPUrl(wapUrl, page) {
    const baseUrl = wapUrl.split('?')[0];
    
    // 修复的分页逻辑：pn = (页码-1) × 30
    if (page === 1) {
      return `${baseUrl}?see_lz=1`;
    } else {
      const pnValue = (page - 1) * 30;
      return `${baseUrl}?see_lz=1&pn=${pnValue}`;
    }
  }

  /**
   * 计算动态延迟
   */
  calculateDelay(page) {
    const base = this.baseDelay;
    const pageOffset = Math.min(page * 100, 2000);
    const random = Math.random() * 1000;
    return base + pageOffset + random;
  }

  /**
   * 提取页面帖子
   */
  extractPagePosts($, page) {
    const posts = [];
    
    // 使用新的页面结构选择器
    $('.list_item').each((index, element) => {
      const $post = $(element);
      
      // 检查是否是楼主发言（优先检查lz属性）
      const $content = $post.find('.content[lz="1"]');
      if ($content.length === 0) {
        // 如果没有lz="1"属性，检查是否有楼主标识
        const hasOPFlag = $post.find('.j_floor_lz.floor_lz_icon').length > 0;
        if (!hasOPFlag) {
          return; // 不是楼主发言，跳过
        }
      }
      
      // 提取内容
      const content = this.extractContentNew($post);
      if (!content || content.length < 3) return;
      
      // 提取楼层号
      const floorNum = this.extractFloorNumberNew($post, index, page);
      
      // 提取时间
      const time = this.extractTimeNew($post);
      
      // 提取作者
      const author = this.extractAuthor($post) || '楼主';
      
      posts.push({
        楼层号: floorNum,
        作者: author,
        内容: content,
        字数: content.length,
        发言时间: time,
        页码: page
      });
    });
    
    return posts;
  }

  /**
   * 提取内容 - 新版本
   */
  extractContentNew($post) {
    // 优先提取标记为楼主的内容
    let content = $post.find('.content[lz="1"]').text().trim();
    
    if (!content) {
      // 如果没有lz属性，尝试其他选择器
      const selectors = ['.content', '.d_post_content', '.post_content', 'p'];
      for (const selector of selectors) {
        content = $post.find(selector).text().trim();
        if (content) break;
      }
    }
    
    return this.cleanText(content);
  }

  /**
   * 提取楼层号 - 新版本
   */
  extractFloorNumberNew($post, index, page) {
    // 尝试从data属性提取
    const dataField = $post.attr('data-field') || $post.attr('data-info');
    if (dataField) {
      try {
        const data = JSON.parse(dataField);
        if (data.floor_num) {
          return parseInt(data.floor_num);
        }
        if (data.fn) {
          return parseInt(data.fn);
        }
      } catch (e) {
        // 忽略JSON解析错误
      }
    }
    
    // 尝试从fn属性提取
    const fn = $post.attr('fn');
    if (fn) {
      return parseInt(fn);
    }
    
    // 尝试从文本中提取楼层号
    const floorText = $post.find('.lou_num, .floor_num, .tail_info, .list_item_time').text();
    const match = floorText.match(/(\d+)楼/);
    if (match) {
      return parseInt(match[1]);
    }
    
    // 估算楼层号
    return (page - 1) * 30 + index + 1;
  }

  /**
   * 提取时间 - 新版本
   */
  extractTimeNew($post) {
    const timeText = $post.find('.list_item_time, .tail_info, .time, .date').text();
    return this.cleanText(timeText) || '未知时间';
  }

  /**
   * 清理和排序帖子
   */
  cleanAndSortPosts(posts) {
    // 去重
    const seen = new Set();
    const unique = posts.filter(post => {
      const key = `${post.楼层号}-${post.内容.substring(0, 30)}`;
      if (seen.has(key)) return false;
      seen.add(key);
      return true;
    });
    
    // 按楼层排序
    return unique.sort((a, b) => a.楼层号 - b.楼层号);
  }

  /**
   * 生成完整报告
   */
  generateCompleteReport(postInfo, posts) {
    const totalWords = posts.reduce((sum, p) => sum + p.字数, 0);
    const storyLine = posts.map(p => p.内容).join('\n\n');
    
    return {
      基本信息: {
        标题: postInfo.title,
        楼主: postInfo.opName,
        帖子ID: postInfo.postId,
        原帖链接: postInfo.originalUrl,
        WAP链接: postInfo.wapUrl,
        楼主发言数: posts.length,
        总字数: totalWords,
        平均字数: posts.length > 0 ? Math.round(totalWords / posts.length) : 0,
        楼层范围: posts.length > 0 ? 
          `${posts[0].楼层号} - ${posts[posts.length-1].楼层号}楼` : '无',
        爬取时间: moment().format('YYYY-MM-DD HH:mm:ss'),
        爬取方式: '增强版多策略完整爬取'
      },
      
      楼主完整故事: storyLine,
      
      楼主发言详细: posts,
      
      质量报告: {
        数据完整性: posts.length > 50 ? '✅ 完整' : posts.length > 20 ? '⚠️ 较完整' : '❌ 不完整',
        去重处理: '✅ 已去重',
        排序: '✅ 按楼层排序',
        总评: posts.length > 50 ? '优秀' : posts.length > 20 ? '良好' : '待改进'
      }
    };
  }

  /**
   * 保存结果
   */
  async saveResults(report, outputName) {
    const timestamp = moment().format('YYYY-MM-DD_HH-mm-ss');
    const cleanTitle = report.基本信息.标题.replace(/[\\/:*?"<>|]/g, '').substring(0, 20);
    const savedFiles = [];

    try {
      await fs.ensureDir('./data');

      // JSON文件
      const jsonFile = `./data/${cleanTitle}_增强完整_${timestamp}.json`;
      await fs.writeJson(jsonFile, report, { spaces: 2 });
      savedFiles.push(jsonFile);

      // 文本文件
      const txtFile = `./data/${cleanTitle}_增强完整故事_${timestamp}.txt`;
      const storyText = this.generateStoryText(report);
      await fs.writeFile(txtFile, storyText, 'utf8');
      savedFiles.push(txtFile);

      console.log(`💾 增强完整版已保存:`);
      savedFiles.forEach(file => console.log(`   📄 ${file}`));

      return savedFiles;

    } catch (error) {
      console.error('❌ 保存失败:', error.message);
      throw error;
    }
  }

  /**
   * 生成故事文本
   */
  generateStoryText(report) {
    const info = report.基本信息;
    const posts = report.楼主发言详细;

    let text = `${info.标题}\n`;
    text += `楼主：${info.楼主} | 生成时间：${info.爬取时间}\n`;
    text += `爬取方式：${info.爬取方式}\n`;
    text += `${'='.repeat(60)}\n\n`;

    text += `📊 楼主发言统计：\n`;
    text += `├─ 发言总数：${info.楼主发言数}楼\n`;
    text += `├─ 总字数：${info.总字数}字\n`;
    text += `├─ 平均字数：${info.平均字数}字/楼\n`;
    text += `├─ 楼层范围：${info.楼层范围}\n`;
    text += `└─ 数据质量：${report.质量报告.总评}\n\n`;

    text += `📖 楼主完整故事：\n\n`;
    
    if (posts.length > 0) {
      posts.forEach(post => {
        text += `【${post.楼层号}楼】${post.内容}\n\n`;
      });
    } else {
      text += `⚠️ 未获取到楼主发言\n\n`;
    }

    text += `${'='.repeat(60)}\n`;
    text += `数据来源：增强版多策略完整爬取器\n`;
    text += `原帖链接：${info.原帖链接}\n`;

    return text;
  }

  /**
   * 工具方法
   */
  convertToWapUrl(pcUrl) {
    if (pcUrl.includes('waptieba.baidu.com')) return pcUrl;
    const match = pcUrl.match(/\/p\/(\d+)/);
    return match ? `https://waptieba.baidu.com/p/${match[1]}` : pcUrl;
  }

  cleanText(text) {
    if (!text) return '';
    return text
      .replace(/看高清大图，打开贴吧App/g, '')
      .replace(/\[.*?\]/g, '')
      .replace(/\s+/g, ' ')
      .trim();
  }
}

/**
 * 主函数
 */
async function main() {
  const args = process.argv.slice(2);
  
  if (args.length === 0) {
    console.log(`
🚀 增强版只看楼主完整爬取器使用说明:

基础用法:
  node src/enhancedOPCrawler.js <帖子URL>

高级用法:
  node src/enhancedOPCrawler.js <帖子URL> [选项]

选项:
  --output <名称>         输出文件名标识
  --pages <数量>          最大爬取页数 (默认: 200)
  --delay <毫秒>          基础延迟时间 (默认: 4000)
  --cookies <文件>        cookies文件路径

示例:
  # 增强完整爬取
  node src/enhancedOPCrawler.js "https://tieba.baidu.com/p/5868510485" --cookies cookies.txt
  
  # 超大规模爬取
  node src/enhancedOPCrawler.js "https://tieba.baidu.com/p/5868510485" --pages 500 --cookies cookies.txt
    `);
    return;
  }

  const postUrl = args[0];
  let outputName = 'enhanced_complete';
  let maxPages = 200;
  let delay = 4000;
  let cookiesFile = '';

  // 解析参数
  for (let i = 1; i < args.length; i++) {
    switch (args[i]) {
      case '--output':
        outputName = args[++i] || outputName;
        break;
      case '--pages':
        maxPages = parseInt(args[++i]) || maxPages;
        break;
      case '--delay':
        delay = parseInt(args[++i]) || delay;
        break;
      case '--cookies':
        cookiesFile = args[++i] || '';
        break;
    }
  }

  try {
    // 读取cookies
    let cookies = '';
    if (cookiesFile && await fs.pathExists(cookiesFile)) {
      cookies = await fs.readFile(cookiesFile, 'utf8');
      console.log(`🍪 已加载cookies: ${cookiesFile}`);
    }

    // 创建增强爬取器
    const crawler = new EnhancedOPCrawler({
      maxPages,
      delay,
      cookies: cookies.trim()
    });

    // 开始爬取
    const startTime = Date.now();
    const result = await crawler.crawlCompleteOP(postUrl, outputName);
    const duration = Math.round((Date.now() - startTime) / 1000);

    console.log(`\n🎉 增强完整爬取完成!`);
    console.log(`   ├─ 楼主: ${result.postInfo.opName}`);
    console.log(`   ├─ 楼主发言: ${result.totalPosts}楼`);
    console.log(`   ├─ 总字数: ${result.totalWords}字`);
    console.log(`   ├─ 保存文件: ${result.savedFiles.length}个`);
    console.log(`   └─ 耗时: ${Math.round(duration/60)}分${duration%60}秒`);

  } catch (error) {
    console.error('\n❌ 增强爬取失败:', error.message);
    process.exit(1);
  }
}

if (require.main === module) {
  main().catch(console.error);
}

module.exports = EnhancedOPCrawler; 